package com.spark;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.ToAnalysis;
import org.apache.commons.io.FileUtils;

import java.io.*;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by TRS on 2017/6/19.
 */
public class Test {

    public static void main(String[] args) throws IOException {

        Integer[] ids = {0, 1, 2, 4, 6, 9};
        List<String> res=new ArrayList<String>();

        for (Integer id : ids) {
            String fileName = "E:\\file\\res\\" + id + ".txt";
            File file = new File(fileName);
            List<String> lines = FileUtils.readLines(file);
            for (String line : lines) {
                String news=line.substring(1,line.length()-1).replaceAll("[^\u4E00-\u9FA5]", "");
                List<Term> terms=ToAnalysis.parse(news).getTerms();

                if(terms.size()>20){
                    String words="";
                    for(Term t:terms){
                        words+=t.getName()+" ";
                    }
                    res.add(id+","+words);

                }
                System.out.println(terms.size());

            }


        }
        File file=new File("E:\\file\\res\\test.txt");
        FileUtils.writeLines(file,"UTF-8",res);


    }


    public static void test1() throws IOException {


        File f = new File("E:\\newx.xml");
        InputStreamReader isr = new InputStreamReader(new FileInputStream(f), "UTF-8");
        BufferedReader br = new BufferedReader(isr);


        BufferedWriter bw = new BufferedWriter(new FileWriter(new File("E:\\new_test.json")));


        String line = "";
        int i = 0;

        while ((line = br.readLine()) != null) {


            try {
                String json = XmlExercise.xml2json(line).replaceAll("\r|\n", "").replaceAll(" ", "");

                //  JSONObject jsonObject = JSON.parseObject(json);
                // String url = jsonObject.getString("url");
                //  URL u = new URL(url);
                // u.getHost();
                // jsonObject.put("host", u.getHost());


                bw.write(json);
                bw.newLine();

                System.out.println(i);

            } catch (Exception e) {

                e.printStackTrace();
            }


            i++;


        }


        bw.close();

    }

    public static void test() throws IOException {

        File f = new File("E:\\file\\sougou-train");
        File[] files = f.listFiles();
        List<String> allFile = new ArrayList<String>();
        for (File subfile : files) {

            allFile.addAll(FileUtils.readLines(subfile));

        }


        File dstFile = new File("E:\\file\\res\\allType.txt");
        File testFile = new File("E:\\file\\test.txt");

        int size = allFile.size();
        int trainSize = (int) (size * 0.9);


        FileUtils.writeLines(dstFile, "UTF-8", allFile);

        System.out.println(allFile.size());
    }
}
